package test

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}


object WordCountPositive {

    def main(args: Array[String]): Unit = {

        val config: SparkConf = new SparkConf().setMaster("local[*]").setAppName("WordCount")

        val sc = new SparkContext(config)

        val lines: RDD[String] = sc.textFile("C:\\Users\\26410\\Desktop\\Review-Positive.txt")

        val words: RDD[String] = lines.flatMap(_.split(" "))

        val wordsToOne: RDD[(String, Int)] = words.map((_, 1))

        val wordToSum: RDD[(String, Int)] = wordsToOne.reduceByKey(_ + _)

        //       wordToSum.sortBy((_._2),false,1).saveAsTextFile("output")

        val total: Long = wordsToOne.count()

        val freRDD: RDD[(String, Double)] = wordToSum.mapValues(x => x.toDouble / total)

        wordToSum.join(freRDD).sortBy(_._2._1, false, 1).saveAsTextFile("output\\Positive")

    }

}
